#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import time
import sys
from pybloom_live import ScalableBloomFilter

reload(sys)
sys.setdefaultencoding('utf-8')


def generate_venom_name():
    '''生成venom数据文件名称'''
    timestamp = time.time()
    timeArray = time.localtime(timestamp)
    time_str = time.strftime("%Y-%m-%d", timeArray)
    timestamp_str = str(timestamp).split('.')[0]
    log_name = './data/1506.log.{}-{}'.format(time_str, timestamp_str)
    time.sleep(1)
    print(log_name)
    return log_name

file_name = 'recruiting_history'
index = 0
mobile_bloom = ScalableBloomFilter(initial_capacity=1000000, error_rate=0.001, mode=2)  # 慢点但是省内存
phone_bloom = ScalableBloomFilter(initial_capacity=1000000, error_rate=0.001, mode=2)  # 慢点但是省内存
email_bloom = ScalableBloomFilter(initial_capacity=1000000, error_rate=0.001, mode=2)  # 慢点但是省内存
other_bloom = ScalableBloomFilter(initial_capacity=1000000, error_rate=0.001, mode=2)  # 慢点但是省内存
extract_bloom = ScalableBloomFilter(initial_capacity=1000000, error_rate=0.001, mode=2)  # 慢点但是省内存
with open(file_name,'r' ) as f ,open('bloom_length', 'w'  ) as log_f:
    ls = []
    for line in f:
        line = line.strip()
        if line:
            dic = json.loads(line)
            contact_ls = dic.get('recruitingContact') or []
            extract_mobile_ls = dic.get('extract_mobile_ls') or []
            for tmp_contact_dic in contact_ls:
                contact = tmp_contact_dic.get('contact','')
                contactType = tmp_contact_dic.get('contactType','')
                _id = "{}_{}".format(contact,contactType)
                if contactType == 0:
                    mobile_bloom.add(_id)
                elif contactType == 1:
                    phone_bloom.add(_id)
                elif contactType == 2:
                    email_bloom.add(_id)
                else:
                    other_bloom.add(_id)
            for i in extract_mobile_ls:
                extract_bloom.add(i)
            dic.pop('recruitingDesc','')
            ls.append(json.dumps(dic))



        if len(ls) == 50000:
            log_name = generate_venom_name()
            with open(log_name,'w' ) as f:
                f.write('\n'.join(ls))
                ls = []
            print('当前bloom长度为：{}'.format(mobile_bloom.count))
            log_f.write('{}\t{}\t{}\t{}\t{}\n'.format(mobile_bloom.count,phone_bloom.count,email_bloom.count,other_bloom.count,extract_bloom.count))
    else:
        if len(ls):
            log_name = generate_venom_name()
            with open(log_name, 'w' ) as f:
                f.write('\n'.join(ls))
            log_f.write('{}\t{}\t{}\t{}\t{}\n'.format(mobile_bloom.count, phone_bloom.count, email_bloom.count,
                                                      other_bloom.count, extract_bloom.count))



